# -*- coding: utf-8 -*-
'''
@author: olivia.dou
Created on: 2018/11/6 15:46
desc:
'''
import logging, time, random, sys, re
import pandas as pd

sys.path.append('..')
import base.globalvars as glo
from proj_spec.inews.news import get_news_tags, news_exist
from proj_spec.inews.common import market_code_dict, get_data_by_criteria
from util.collection_util import random_select_from_list
from util.jsonmatch import jsonmatch
from util.string_util import random_partial_string
from util.db_util import execute_query


tag_type_label_dict = {1: ("Event", "事件"), 2: ("DictIndustry", "行业"), 3: ("Concept", "概念"), 7: ("DictProduct", "产品")}


def get_stock_list(mkt_type):

    if mkt_type != '4': # 除了三板，都加了LT,ST,SST限制
        query = "MATCH (n:Company) where n.mkt_code in %s and n.ls_code in ['LT','ST','SST'] RETURN n._id, n.code, n.abbr_en, n.tick, n.abbr_szh, n.name_szh, n.name_en, n.csfid" \
                % market_code_dict[mkt_type]
    else:
        query = "MATCH (n:Company) where n.mkt_code in %s RETURN n._id, n.code, n.abbr_en, n.tick, n.abbr_szh, n.name_szh, n.name_en, n.csfid"\
                %market_code_dict[mkt_type]

    with glo.get_value("neo4j_driver").session() as session:
        results = session.run(query)
        session.sync()
    return results


def get_random_stock(mkt_type, return_field, complete_str = True):
    resultset = get_stock_list(mkt_type)
    #logging.debug("Keys: %s"%repr(resultset.single().keys())) # Keys: ['n']
    #logging.debug("Values: %s" % repr(resultset.single().values())) #Values: [<Node id=438183 labels={'Company'} properties={'mkt_code': '1012', 'ls_code': 'LT', 'code': '300694_SZ_EQ', 'abbr_en': 'Lihu', 'abbr_py': 'lhgf', 'cru': '000558', 'mkt_en': 'SZSE ChiNext', 'cov': 1, 'tick': '300694', 'cdtid': '913202006079522354', 'orgid': '10106014', 'mkt_abbr': 'SZ', 'abbr_szh': '蠡湖股份', 'mkt_szh': '深圳证券交易所创业板', 'org_id': '10106014', 'name_szh': '蠡湖股份', 'name': '蠡湖股份', 'csfid': 'CSF0000040848', '_id': '5be250858c64101d0c449221', 'upu': '000558', 'name_en': 'Lihu'}>]
    if return_field!="name_en" and return_field!="abbr_en":
        field_list = [record['n.' + return_field] for record in resultset] #指定市场以return_field字段组成的list
    else:  # 先在inews表里取英文名，若无再取neo4j的
        tuple_list = [(record['n.%s'%return_field], record['n.name_szh']) for record in resultset]
        field_list = [ele[0] for ele in tuple_list]
        name_list = [ele[1] for ele in tuple_list]
    if len(field_list)>0:
        rand_idx = random.randint(0, len(field_list) - 1)
        # 英文名先取Inews库里的值，如果为空则仍取Neo4j的值
        if return_field == "name_en":
            result = get_data_by_criteria("nlp_news_eng_label", "name = '%s' order by upt desc"%name_list[rand_idx], "word")
            if result!=((),) and result[0][0]!="":
                result = result[0][0]
            else:
                result = field_list[rand_idx]
        else:
            result = field_list[rand_idx]
        if complete_str:
            return result
        else:
            return random_partial_string(result)
    else:
        return ''

# Todo: 改成参数化的查询
def get_company_id(company_name):
    # session = glo.get_value("neo4j_driver").session()
    with glo.get_value("neo4j_driver").session() as session:
        # with session.begin_transaction() as tx:
        result = session.run('MATCH (n:Company{name:"%s"}) RETURN n._id LIMIT 1' % company_name).single().value()
        # result = tx.run('MATCH (n:Company{name:"%s"}) RETURN n._id LIMIT 1' % company_name).single().value()
        # session.sync()
    return result

def get_stock_code_list(mktCode):
    '''获取指定市场股票代码列表，设置全局变量'''
    sql = "SELECT code,abbr FROM dict_company WHERE mkt = 'A'"
    resultset = execute_query(sql)
    #logging.debug("A stock code list: %s"%repr(resultset))
    glo.set_value('A_Stock_List',resultset)
    return resultset #tuple: (('002054_SZ_EQ', '德美化工'), ('002055_SZ_EQ', '得润电子'), ('002056_SZ_EQ', '横店东磁'),...)

def get_random_A_stock(n_result=1):
    '''获取随机的A股股票,id为Neo4j数据库中的id字段，不是股票代码'''
    found = []

    while len(found) < n_result:
        # random_code = get_random_A_stock_code()
        # session = glo.get_value("neo4j_driver").session()
        with glo.get_value("neo4j_driver").session() as session:
            # with session.begin_transaction() as tx:
            # 只包括已上市未退市的
            result = session.run("MATCH (m:Company) WHERE m.code = '%s' and m.ls_code in ['LT','ST','SST'] RETURN m LIMIT 1")
            # result = tx.run("MATCH (m:Company) WHERE m.code = '%s' and m.ls_code in ['LT','ST','SST'] RETURN m._id,m.code,m.abbr_szh LIMIT 1"%random_code)
            # session.sync()
            if result is not None:
                found.append(result)
    return found


def get_random_A_stock_id(n_result=1):
    result_list = get_random_A_stock()  # 结果是一个BoltStatementResult对象的list
    stock_ids = [record['m._id'] for records in result_list for record in records]
    return ','.join(stock_ids)


def get_random_tag(tag_type):
    # session = glo.get_value("neo4j_driver").session()
    if tag_type == 7:
        query = "MATCH (n:%s) where n._id is not null and not (n.valid='0' or n.code starts with '0x') RETURN n.name" % \
                tag_type_label_dict[tag_type][0]
    else:
        query = "MATCH (n:%s) where n._id is not null RETURN n.name" % tag_type_label_dict[tag_type][0]
    with glo.get_value("neo4j_driver").session() as session:
        # with session.begin_transaction() as tx:
        results = session.run(query)
        # results = tx.run(query)
        # session.sync()
    tags = [record['n.name'] for record in results]
    idx = random.randint(0, len(tags) - 1)
    return tags[idx]


def generate_investment_list(n_result=1):
    """生成保存搜索条件接口/api/mapSearchCriteria/save里的投资组合内容"""
    result_list = get_random_A_stock(n_result)
    code_list = [{"abbr": "%s  ( %s )" % (record['m.abbr_szh'], record['m.code']), "companyId": record['m._id'],
                  "checked": random.randint(0, 1)} for records in result_list for record in records]
    return code_list


def generate_tag_list(n_result):
    tag_types = list(tag_type_label_dict.keys())
    tag_list = []
    for i in range(n_result):
        tag_type_index = random.randint(0, len(tag_types) - 1)
        tag_type = tag_types[tag_type_index]
        tag_name = get_random_tag(tag_type)
        retain = random.randint(0, 1)
        tag_list.append({"tagName": "%s   (%s)" % (tag_name, tag_type_label_dict[tag_type][1]),
                         "tabId": str(tag_type) + '_' + tag_name, "retain": retain,
                         "filtered": random.randint(0, 1) if retain == 0 else 0})
        return tag_list


def convert_to_neo_string(str_list):
    new_list = ["`%s`" % e for e in str_list]
    return '|'.join(new_list)


# def get_stockid_with_criteria(start_time, end_time, emotions=[0, 1, 2], categoryIds=[1, 2, 3, 4, 5], filterLevel="", label_str="",
def get_stockid_with_criteria(start_time, end_time, emotions=[0, 1, 2], categoryIds=[1, 2, 3, 4, 5, 6, 7],
                              n_result=1, random_selection=1, return_newsid=False, isStockReview=""):

    tag_dict = {}
    '''
    if label_str!= "":
        mark = label_str[0]
        if mark != '+' and mark != '-':
            logging.exception("labels 必须以'+'或'-'开始")
            #return
        else:
            labels = [filter for filter in label_str[1:].split(',')]
            filter_include = 1 if mark =='+' else -1
            tag_dict = get_type_tag_dict(labels)
    else:
        filter_include = 0
    '''

    emotion_strlist = [str(emo) for emo in emotions]
    # typeid_list = get_relation_ids(categoryIds)
    # typeid_list_extend = typeid_list + get_relation_ids([6])
    typename_list = []
    relation_category_dict = get_relation_category_dict()
    for c in categoryIds:
        if c!=7 and c!=6: # 非竞争关系也非新闻直接提及
            typename_list.extend(relation_category_dict[c])
    typename_strlist = convert_to_neo_string(typename_list)
    # typename_list_extend = typename_list + relation_category_dict[6]
    # typename_strlist_extend = convert_to_neo_string(typename_list_extend)
    news_relations_list = convert_to_neo_string(relation_category_dict[0]) # 新闻提及的关系列表

    query_1_degree = "match path=(a:Company)-[r]-(n:News) where n.time>='%s' and n.time<='%s' and n.emotion in %s " \
                     "and exists(a.abbr_szh) and a.ls_code in ['LT','ST','SST'] return n.csfid as n_csfid,a._id as a_id limit 1000" \
                     % (start_time, end_time, emotion_strlist)

    query_2_degree = "match path=(a:Company)-[r1:%s]-(b)-[nr:%s]-(n:News) where n.time>='%s' and n.time<='%s' " \
                     "and not(b:Company and b._id=a._id) and not(b:DictProduct " \
                     "and (b.valid = '0' or b.code starts with '0x')) and (r1.rpt is null or (type(r1)='主营产品' and r1.isNew=true) " \
                     "or (type(r1)<>'主营产品' and r1.rpt='%s')) and n.emotion in %s and a.ls_code in ['LT','ST','SST'] " \
                     "return n.csfid as n_csfid, a._id as a_id limit 1000" \
                     % (typename_strlist, news_relations_list, start_time, end_time, glo.get_value("rpt_time"),
                        emotion_strlist)

    if 3 in categoryIds and 7 not in categoryIds:
        # 减去竞争关系的结果
        sub_competition = " and path<>(a:Company)-[r1:主营产品]-(b)-[r2:主营产品]-(c:Company)-[nr:`提及（新闻和公司）`]-(n:News)"
    else:
        sub_competition = ""

    query_3_degree_competition = "match path=(a:Company)-[r1:%s]-(b)-[r2:%s]-(c)-[nr:%s]-(n:News) where n.time>='%s' and n.time<='%s' " \
                                 "and not(b:Company and b._id=a._id) and not(b:DictProduct and (b.valid = '0' or b.code starts with '0x'))  " \
                                 "and not(c:Company and c._id=a._id) and not(c:DictProduct and (c.valid = '0' or c.code starts with '0x'))  " \
                                 "and (r1.rpt is null or r1.isNew=true) and (r2.rpt is null or r2.isNew=true) " \
                                 "and n.emotion in %s and a.ls_code in ['LT','ST','SST'] return n.csfid as n_csfid, a._id as a_id limit 1000" \
                                 % ("主营产品", "主营产品", news_relations_list,start_time, end_time, emotion_strlist)

    query_3_degree = "match path=(a:Company)-[r1:%s]-(b)-[r2:%s]-(c)-[nr:%s]-(n:News) where n.time>='%s' and n.time<='%s' " \
                     "and not(b:Company and b._id=a._id) and not(b:DictProduct and (b.valid = '0' or b.code starts with '0x'))  " \
                     "and not(c:Company and c._id=a._id) and not(c:DictProduct and (c.valid = '0' or c.code starts with '0x'))  " \
                     "and (r1.rpt is null or (type(r1)='主营产品' and r1.isNew=true) or (type(r1)<>'主营产品' and r1.rpt='%s')) " \
                     "and (r2.rpt is null or (type(r2)='主营产品' and r2.isNew=true) or (type(r2)<>'主营产品' and r2.rpt='%s')) " \
                     "and n.emotion in %s and a.ls_code in ['LT','ST','SST'] %s return n.csfid as n_csfid, a._id as a_id limit 1000" \
                     % (typename_strlist, typename_strlist, news_relations_list,
                        start_time, end_time, glo.get_value("rpt_time"), glo.get_value("rpt_time"), emotion_strlist, sub_competition)

    if categoryIds == [7]:
        statements = [query_3_degree_competition]
    elif 6 not in categoryIds:
        statements = [query_2_degree, query_3_degree]
    else:
        statements = [query_1_degree, query_2_degree, query_3_degree]

    all_results = []  # id可以重复，即同时满足多条查询语句的股票中选可能更高
    # session = glo.get_value("neo4j_driver").session()
    with glo.get_value("neo4j_driver").session() as session:
        # with session.begin_transaction() as tx:
        for statement in statements:
            result = session.run(statement)
            # result = tx.run(statement)
            # session.sync()
            # results.extend([record['a_id'] for record in result if news_exist(record['n_csfid'],filter_include,tag_dict)])
            news_set = set()  # 避免对同一新闻id重复查询
            result_list = []
            for record in result:
                news_set.add(record['n_csfid'])
                result_list.append((record['a_id'], record['n_csfid']))  # 迭代完一次之后result里没有元素，因此需要在一次迭代中完成数据收集
            logging.debug("news_set:%s" % repr(news_set))
            # news_exist_dict = {csfid:news_exist(csfid,filter_include,tag_dict) for csfid in news_set}
            news_exist_dict = {csfid: news_exist(csfid, isStockReview) for csfid in news_set}
            all_results.extend([record for record in result_list if news_exist_dict[record[1]] == True])

    if return_newsid == True:
        return all_results
    # stock_ids = [result[0] for result in all_results]
    # news_ids = [result[1] for result in all_results]
    result_stockids = []

    if len(all_results) > 0:
        if random_selection == 1:
            for i in range(min(len(all_results), n_result)):
                rand_idx = random.randint(0, len(all_results) - 1)
                result_stockids.append(all_results[rand_idx][0])
        else:
            result_stockids.extend(all_results[:n_result][0])
        return ",".join(result_stockids)
    else:
        return ""


def get_stockid_tags_with_criteria(start_time, end_time, emotions=[0, 1, 2], categoryIds=[1, 2, 3, 4, 5],
                                   n_stock=1, tag_types=[1, 2, 3, 7], tag_action=1):
    """按顺序获取stockid(非随机)，之后在对应新闻的标签中随机选取1~20个进行组合"""
    results = get_stockid_with_criteria(start_time, end_time, emotions, categoryIds, n_stock,
                                        random_selection=0, return_newsid=True)
    results_frm = pd.DataFrame(results, columns=["stockid", "newsid"])
    tag_dict = {key: [] for key in tag_types}  # 存放results中找到的标签字典
    if len(results_frm) > 0:
        news_set = set([record[1] for record in results])
        stockids = []

        class Getoutofloop(Exception):
            pass

        try:
            for news in news_set:
                for tag_type in tag_types:
                    tag_list = get_news_tags(tag_type, news)
                    if len(tag_list) > 0:
                        tag_dict[tag_type].extend(tag_list)
                        stockids.append(results_frm[results_frm['newsid'] == news]['stockid'].iloc[0])  # 同一个新闻添加一个股票即可
                        if len(stockids) >= min(n_stock, 10):
                            raise Getoutofloop()  # 跳出外层循环
        except Getoutofloop:
            pass

        # result_tag_types = random_select_from_list(list(tag_dict.keys()))  # 随机选取测试用的tag类型
        result_tag_types = tag_types
        list_of_taglist = [[str(tag_type) + '_' + tag for tag in tag_dict[tag_type] if tag_dict[tag_type] != []] for
                           tag_type in result_tag_types]
        tag_list = list(set([item for sublist in list_of_taglist for item in sublist]))[:20]  # 去重后取前20个
        if len(tag_list) != 0:
            if tag_action == 1:
                mark = '+'
            else:
                mark = '-'
            tag_str = mark + ','.join(tag_list)

        else:
            tag_str = ""
        stockid_list = list(set(stockids))
        stockidstr = ",".join(stockid_list)
        return stockidstr, tag_str
    else:
        return "", ""

# Todo: 改成参数化查询
def get_relation_type_id(relation_name):
    # session = glo.get_value("neo4j_driver").session()
    with glo.get_value("neo4j_driver").session() as session:
        # with session.begin_transaction() as tx:
        result = session.run(
            "MATCH (n:BaseRelationType{name:'%s'}) where n.level = 2 RETURN n.id LIMIT 100" % relation_name).single()
        # result = tx.run("MATCH (n:BaseRelationType{name:'%s'}) where n.level = 2 RETURN n.id LIMIT 100" % relation_name).single()
        # session.sync()
        if result is not None:
            return result.value()
        else:
            logging.error("Cannot find matching for %s" % relation_name)


def get_relation_category_dict():
    return {
        1: ["子公司", "重大股权投资", "母公司（公司和公司）", "母公司（人和公司）", "实际控制人（人和公司）",
            "实际控制人（公司和公司）", "前10大股东（公司和公司）", "前10大股东（人和公司）", "前10大流通股东（公司和公司）",
            "前10大流通股东（人和公司）"],
        2: ["重要客户—主要客户（人和公司）", "重要客户—主要客户（公司和公司）", "重要客户—应收帐款（人和公司）", "重要客户—应收帐款（公司和公司）", "供应商（人和公司）", "供应商（公司和公司）",
            "重要其他应付款（公司和公司）", "重要其他应付款（人和公司）"],
        3: ["下位产品", "生产设备", "销售渠道", "生产环境", "辅助设备", "生产原料", "产品/业务", "辅助原料", "技术服务",
            "相关公司", "主营产品"],
        4: ["关联交易—应收账款（人和公司）", "关联交易—应收账款（公司和公司）", "关联交易—销售（人和公司）",
            "关联交易—销售（公司和公司）", "关联交易—应付账款（人和公司）", "关联交易—应付账款（公司和公司）",
            "关联交易—采购（人和公司）", "关联交易—采购（公司和公司）", "关联交易—担保（人和公司）",
            "关联交易—担保（公司和公司）", "关联交易—租赁（人和公司）", "关联交易—租赁（公司和公司）",
            "关联交易—租赁（人和人）", "关联交易—资金拆借（人和公司）", "关联交易—资金拆借（公司和公司）",
            "关联交易—资金拆借（人和人）"],
        5: ["重大非股权投资", "董监高", "担保（人和公司）", "担保（公司和公司）"],
        0: ["提及（新闻和公司）", "提及（新闻和产品）", "提及（新闻和事件）", "提及（新闻和概念）", "提及（新闻和人物）"]}
    # 推导关系“提及（新闻和行业）"”和“证券投资”(292)这两个细分关系排除。


def get_relation_type_dict():
    if glo.get_value("relation_type_dict") is None:
        relation_category_dict = get_relation_category_dict()
        relation_type_dict = {}
        for k in relation_category_dict.keys():
            relation_type_dict[k] = [get_relation_type_id(type_name) for type_name in relation_category_dict[k]]

        glo.set_value("relation_type_dict", relation_type_dict)

    return glo.get_value("relation_type_dict")


def get_relation_ids(categoryIdList):
    """根据关系大类(category)id获取关系的typeId"""
    relation_type_dict = get_relation_type_dict()

    relation_type_ids = []
    for id in categoryIdList:
        if id!=7 and id!=6:
            relation_type_ids.extend(relation_type_dict[id])  # 构成一个新的单层列表

    return relation_type_ids


def get_type_tag_dict(tagList):
    '''tagList: 符号相同的tag list'''
    tagDict = {}

    for filtertag in tagList:
        try:
            [type, tag] = filtertag.split('_')  # 以_切分tag类型和内容
        except Exception as e:
            logging.exception(e)
            logging.debug("filtertag: %s" % filtertag)
        if int(type) not in tagDict.keys():
            tagDict[int(type)] = []
        tagDict[int(type)].append(tag)
    logging.debug("tagDict: %s" % repr(tagDict))

    return tagDict

def get_stock_en_dict(field_name):
    global_en_dict = glo.get_value("%s_en_dict"%field_name)
    if global_en_dict is None:
        with glo.get_value("neo4j_driver").session() as session:
            statement_result = session.run("MATCH (n:Company) WHERE n.%s_szh is not null and n.%s_en is not null "
                                           "RETURN n.%s_szh, n.%s_en union "
                                           "MATCH (n:OrgHkOtc) WHERE n.%s_szh is not null and n.%s_en is not null "
                                           "RETURN n.%s_szh, n.%s_en"%(field_name,field_name,field_name,field_name,field_name,field_name,field_name,field_name))
            session.sync()
            result_list = [(record['n.%s_szh'%field_name], record['n.%s_en'%field_name]) for record in statement_result] #迭代器内容只能获取一次
            en_dict_neo4j = {record[0]:record[1] for record in result_list}
            key_word_tuples = execute_query("SELECT name,word FROM nlp_news_eng_label WHERE label='com' AND name in %s"
                                             %repr(tuple(en_dict_neo4j.keys())))
            key_word_dict = {record[0]: record[1] for record in key_word_tuples if
                              record[1] is not None and record[1] != ''}

            # 英文名字典针对所有股票
            eng_dict = {
            item[0]: key_word_dict[item[0]] if item[0] in key_word_dict.keys() else item[1] for item in en_dict_neo4j.items()}
            glo.set_value("%s_en_dict"%field_name,eng_dict)
            return eng_dict
    else:
        return global_en_dict

def validate_search_stock_result(post_data, res_json):
    """验证股票搜索接口返回数据，英文名称搜索先查nlp_news_eng_label表，如果无结果再查neo4j name_en，继而abbr_en"""

    logging.debug("response json: %s"%repr(res_json))
    keyword = post_data['params[name]']
    if keyword == "":
        logging.exception("未输入关键字")
        return False

    #lang = post_data['params[lang]'] 中英文接口返回内容一致

    if len(res_json['message'])==0:
        assert 1==0,"搜索股票没有返回结果：%s"%keyword
    keyword_l = keyword.lower()
    for msg in res_json['message']:
        assert keyword_l in msg['abbr'].lower() or keyword_l in msg['code'].lower() or keyword_l in msg['nameen'].lower() or keyword_l in msg['namezh'].lower(),"返回股票%s中未包含搜索关键字%s"%(msg['abbr'],keyword)

    return True


def validate_search_tag_result(post_data, res_json):
    tag_name = post_data['params[name]']
    tag_type = post_data['params[type]']

    if all(ord(c) < 128 for c in tag_name):  #tag名为英文
        resultset = execute_query("SELECT name from nlp_news_eng_label WHERE word = '%s'"%tag_name)
        if resultset != []:
            tag_name = resultset[0][0]
        else:  # 输入的英文在翻译表里没有对应的中文内容
            pass
            # if response.json == {"code": "200", "type": "SUCCESS", "message": []}:
            #     logging.warning("输入内容没有对应的标签词")
            #     return True
            # else:
            #     return False

            #Todo: 下一版本标签同义词

    # session = glo.get_value("neo4j_driver").session()
    with glo.get_value("neo4j_driver").session() as session:
        # with session.begin_transaction() as tx:
        if tag_type == 7:  # 产品
            query = "MATCH (n:%s) where (n.name =~ '.*%s.*' or  n.name_en =~ '.*%s.*') and n._id is not null and not (n.valid='0' or n.code starts with '0x') " \
                    "RETURN n.csfid,n._id,n.name,n.name_en as nameen order by length(n.name),n.csfid" \
                    % (tag_type_label_dict[tag_type][0], tag_name, tag_name)
        else:
            query = "MATCH (n:%s) where n.name =~ '.*%s.*' and n._id is not null RETURN n.csfid,n._id,n.name,n.name_en order by length(n.name) " \
                    "union MATCH (n:%s) where n.name_en =~ '.*%s.*' and n._id is not null RETURN n.csfid,n._id,n.name,n.name_en order by length(n.name)" \
                    % (tag_type_label_dict[tag_type][0], tag_name, tag_type_label_dict[tag_type][0], tag_name)
        results = session.run(query)
        # results = tx.run(query)
        # session.sync()
        #Todo:英文名
        msgs = [{"id": record['n._id'], "nameen":"","name": record['n.name'], "csfid": str(record['n.csfid']),
                 "label": tag_type_label_dict[tag_type][0],
                 "type": tag_type} for record in results]  # Product返回的csfid是数字格式，需要转换
        json_obj = {"code": "200", "type": "SUCCESS", "message": msgs}

    logging.debug(json_obj)
    logging.debug(res_json)

    if jsonmatch(json_obj, res_json):
        return True
    else:
        logging.exception("预期的json返回结果和实际不符")
        return False

def isStockReview(csfid):
    is_stock_review = execute_query("SELECT is_stock_review from nlp_news_additional WHERE oid = %d"%int(csfid))
    if is_stock_review !=[]:
        return is_stock_review
    else:
        return None

def matchCounterPart(node_id, input_ids):
    """判断输入的node_id是否为ids列表中的元素在异地上市的对应股票"""
    for id in input_ids:
        query = "MATCH (n) where (n:Company or n:OrgHkOtc) and n._id='%s' with n.csfid as csfid MATCH (m) where  " \
                "(m:Company or m:OrgHkOtc) and m.csfid=csfid and m._id<>'%s' return m._id"%(id,id)
        with glo.get_value("neo4j_driver").session() as session:
            results = session.run(query)
            session.sync()
            counterpart_ids = [record['m._id'] for record in results]
            if node_id in counterpart_ids:
                return True

    return False

def find_competition_relation(nodes, num_main_product):
    nodes = pd.DataFrame(nodes)
    companies = nodes[nodes['type'] == 6]
    # news = nodes[nodes['type'] == 8]
    products = nodes[nodes['type'] == 7]

    if len(nodes)<4 or len(products) != 1 or len(companies)<2 or num_main_product< 2:
        return False

    return True

def get_csfid(news_id):
    """通过新闻的id从neo4j库获取csfid"""
    query = "MATCH (n:News) where n._id='%s' RETURN n.csfid"%news_id
    with glo.get_value("neo4j_driver").session() as session:
        result = session.run(query).single().value()
        session.sync()
    return result


def validate_search_news_result(post_data, res_json):

    tag_dict = {}
    # 提取标签参数
    if 'params[labels]' in post_data.keys() and post_data['params[labels]'] != "":
        mark = post_data['params[labels]'][0]
        if mark != '+' and mark != '-':
            logging.exception("labels参数必须以+或者-开始")
        else:
            labels = [label for label in post_data['params[labels]'][1:].split(',')]
            tag_dict = get_type_tag_dict(labels)

    if len(res_json['message']) == 0:
        logging.warning("返回新闻列表为空")
        # return True
        return False  # 为了查看结果方便，把验证结果设为False

    ids = set([id for id in post_data['params[ids]'].split(',')])  # 随机产生的股票代码有可能重复？
    if ids == set({''}):  # 没有输入股票代码
        assert res_json['type'] == 'INVALID_PARAMETERS', '未输入股票代码时应返回无效参数'
        logging.warning("输入股票ID为空")
        return False  # 目前把没有数据覆盖到的情况都标记为Fail

    st = time.strptime(post_data['params[st]'], '%Y%m%d%H%M%S')
    et = time.strptime(post_data['params[et]'], '%Y%m%d%H%M%S')
    emotions = post_data['params[emotions]'].split(',')
    typeCategories = [int(type) for type in post_data['params[typeIds]'].split(',')]
    if 0 not in typeCategories:
        typeCategories.append(0)  # 加上新闻推荐关系大类
    #filterLevel = post_data['params[filterLevel]']
    param_is_stock_review = post_data['params[isStockReview]']

    # relation_ids = get_relation_ids(typeIds).extend([283,284,286,287,288])#返回值为None，不能用链式  推导关系“新闻提及行业”(285)和“证券投资”这两个细分关系排除。
    relation_type_dict = get_relation_type_dict()  #
    rtypid_typid_dict = {}  # 根据推荐关系id查找大类id
    for key, value in relation_type_dict.items():
        for v in value:
            rtypid_typid_dict[v] = key
    relation_ids = get_relation_ids(typeCategories)
    # 需要加入新闻提及关系
    relation_ids.extend(get_relation_ids([0]))
    # 含竞争关系的应加入主营产品
    if 7 in typeCategories:
        main_product_typeid = get_relation_type_id("主营产品")
        if main_product_typeid not in relation_ids:
            relation_ids.append(main_product_typeid)

    previous_time = time.time()

    found_emotions = []  # 需检查是否每个选定的emotion都搜到了对应的新闻，否则需给出警告，查看是否数据限制
    found_stocks = []  # 需检查是否每个选定的投资组合项目都搜到了对应的新闻，否则需给出警告，查看是否数据限制
    found_typeIds = []  # 需检查是否每个选定的推荐关系大类都搜到了对应的新闻，否则需给出警告，查看是否数据限制

    no_exception = True

    if isinstance(res_json['message'], str):  # error message
        logging.error("code: %s" % res_json['code'])
        logging.error("message: %s" % res_json['message'])
        logging.error("type: %s" % res_json['type'])
        return False
    try:
        for msg in res_json['message']:
            assert msg['summary'] is not None, "summary缺失: %s"%msg['title']
            msg_tim = time.strptime(msg['tim'], '%Y-%m-%d %H:%M:%S')
            assert st <= msg_tim <= et, '新闻时间不在选定范围内: 标题：%s, id: %s, 时间：%s' % (msg['title'], msg['id'], repr(msg_tim))

            if post_data['params[sort]'] == '1':
                assert time.mktime(msg_tim) <= previous_time, '新闻时间排序不正确: %s, previous_time: %s, msg_tim:%s' % (msg['title'], time.localtime(previous_time), msg_tim)  # 验证新闻按时间降序排列
            previous_time = time.mktime(msg_tim)

            if msg['pos'] not in found_emotions:
                found_emotions.append(msg['pos'])
            assert msg['pos'] in emotions, '新闻情感不在选定范围内: 标题：%s，情感：%s' % (msg['title'], msg['pos'])
            loc_stock = 0
            # if len(msg['paths']['nodes'])==2:  #新闻直接提及
            #     logging.debug("新闻直接提及： %s"%msg['title'])
            for node in msg['paths']['nodes']:
                #Todo: 返回的股票可以是被搜索股票在异地上市的counterpart
                # 改每次都查询counterpart为只在股票id不在列表中时才查询counterpart
                #matched = matchCounterPart(node['id'], ids)
                if node['type']==6 and (node['id'] in ids or matchCounterPart(node['id'], ids)):  # id在请求的股票id列表中或者是异地上市的股票
                    loc_stock = 1
                    if node['id'] not in found_stocks:
                        found_stocks.append(node['id'])
                    if not node['id'] in ids:  # 返回的股票是被搜索股票在异地上市的counterpart时，searchStock值须为True，否则UI上该节点非高亮显示
                        assert node['searchStock']==True, "股票为被搜索股票异地上市的counterpart, searchStock值须为True. title: %s"%msg['title']
                    break
                if node['type'] == 8:  # 新闻
                    csfid = get_csfid(node['id'])
                    if param_is_stock_review!="":
                        assert isStockReview(csfid) == param_is_stock_review,"新闻股评类型与所选不一致，标题：%s, csfid: %s"%(msg['title'],csfid)

            assert loc_stock == 1, '股票不在选定范围内：%s,%s' % (msg['title'], msg['paths']['nodes'][0])

            # 验证推荐关系
            found_main_product = 0 # 新闻的关系中，主营产品关系的计数

            for relation in msg['paths']['relationships']:
                if rtypid_typid_dict[relation['type']] not in found_typeIds:
                    found_typeIds.append(relation['type'])
                assert relation['type'] in relation_ids, '推荐关系类型不在选定范围内：%s,%s' % (msg['title'], relation['type'])
                if relation['typeNamezh']=='主营产品':
                    found_main_product += 1

            found_competition = find_competition_relation(msg['paths']['nodes'],found_main_product)

            if found_competition:
                found_typeIds.append('7')

            # 含产业链关系而不含竞争关系的情况下应排除竞争关系
            if '3' in post_data['params[typeIds]'] and '7' not in post_data['params[typeIds]']:
                assert not found_competition, "竞争关系应排除： %s" % msg['title']

            if '6' not in post_data['params[typeIds]']:
                assert len(msg['paths']['nodes']) > 2, '新闻直接提及应被过滤： :s' % msg['title']
            elif  post_data['params[typeIds]'] == '6': #仅新闻提及关系
                assert len(msg['paths']['nodes']) == 2, '应仅包含新闻直接提及关系： :s' % msg['title']
            # filtertag_types = [filter[0] for filter in filterTags] #第一位数字为类型

            news_id = msg['oid']
            if tag_dict != {}:
                if mark == '-':  # 过滤
                    for tagtype in tag_dict.keys():
                        msg_tags = get_news_tags(tagtype, news_id)
                        for tag in msg_tags:
                            assert tag not in tag_dict[tagtype], '新闻标签在过滤范围内——标题：%s,类型:%s, 标签： %s' % (
                            msg['title'], tagtype, tag)
                else:  # 保留
                    found_dict = {type: {tag: 0 for tag in tag_dict[type]} for type in tag_dict.keys()}
                    for tagtype in tag_dict.keys():
                        msg_tags = get_news_tags(tagtype, news_id)
                        for tag in msg_tags:
                            if tag in found_dict[tagtype].keys():
                                found_dict[tagtype][tag] = 1

                    for type in found_dict.keys():
                        if not (max(found_dict[type].values()) == 1):  # 新闻中只需包含任一目标tag

                            assert False, '新闻title: %s，未包含%s类型标签' % (msg['title'], type)

            # Todo: nodes[0]是否均为所选股票？若是，则nodes[-1]的id,name应和msg的id,'title'一致

        if len(found_emotions) < len(emotions):
            logging.warning(
                "搜索结果中未包含所有选定的新闻情感，请检查是否数据限制，选定的新闻情感： %s， 结果中的新闻情感：%s" % (repr(emotions), repr(found_emotions)))
            no_exception = False

        if len(found_stocks) < len(ids):
            logging.warning(
                "搜索结果中未包含投资组合中所有项目，请检查是否数据限制，选定的投资组合： %s， 结果中的公司id：%s" % (repr(ids), repr(found_stocks)))
            no_exception = False
        if len(found_typeIds) < len(typeCategories) + 1:  # 接口中给出的推荐关系大类，加上新闻提及类
            logging.warning(
                "搜索结果中未包含所有选定的推荐关系类型，请检查是否数据限制，选定的推荐关系类型： %s， 结果中的推荐关系类型：%s" % (
                repr(relation_ids), repr(found_typeIds)))
            no_exception = False

    except Exception as e:
        logging.exception(e)
        no_exception = False

    return no_exception
